{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# 2 actions\n",
    "ACTIONS = [0, 1]\n",
    "\n",
    "# each transition has a probability to terminate with 0\n",
    "TERMINATION_PROB = 0.1\n",
    "\n",
    "# maximum expected updates\n",
    "MAX_STEPS = 20000\n",
    "\n",
    "# epsilon greedy for behavior policy\n",
    "EPSILON = 0.1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# break tie randomly\n",
    "def argmax(value):\n",
    "    max_q = np.max(value)\n",
    "    return np.random.choice([a for a, q in enumerate(value) if q == max_q])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Task:\n",
    "    # @n_states: number of non-terminal states\n",
    "    # @b: branch\n",
    "    # Each episode starts with state 0, and state n_states is a terminal state\n",
    "    def __init__(self, n_states, b):\n",
    "        self.n_states = n_states\n",
    "        self.b = b\n",
    "\n",
    "        # transition matrix, each state-action pair leads to b possible states\n",
    "        self.transition = np.random.randint(n_states, size=(n_states, len(ACTIONS), b))\n",
    "\n",
    "        # it is not clear how to set the reward, I use a unit normal distribution here\n",
    "        # reward is determined by (s, a, s')\n",
    "        self.reward = np.random.randn(n_states, len(ACTIONS), b)\n",
    "\n",
    "    def step(self, state, action):\n",
    "        if np.random.rand() < TERMINATION_PROB:\n",
    "            return self.n_states, 0\n",
    "        next_ = np.random.randint(self.b)\n",
    "        return self.transition[state, action, next_], self.reward[state, action, next_]\n",
    "\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Evaluate the value of the start state for the greedy policy\n",
    "# derived from @q under the MDP @task\n",
    "def evaluate_pi(q, task):\n",
    "    # use Monte Carlo method to estimate the state value\n",
    "    runs = 1000\n",
    "    returns = []\n",
    "    for r in range(runs):\n",
    "        rewards = 0\n",
    "        state = 0\n",
    "        while state < task.n_states:\n",
    "            action = argmax(q[state])\n",
    "            state, r = task.step(state, action)\n",
    "            rewards += r\n",
    "        returns.append(rewards)\n",
    "    return np.mean(returns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# perform expected update from a uniform state-action distribution of the MDP @task\n",
    "# evaluate the learned q value every @eval_interval steps\n",
    "def uniform(task, eval_interval):\n",
    "    performance = []\n",
    "    q = np.zeros((task.n_states, 2))\n",
    "    for step in tqdm(range(MAX_STEPS)):\n",
    "        state = step // len(ACTIONS) % task.n_states\n",
    "        action = step % len(ACTIONS)\n",
    "\n",
    "        next_states = task.transition[state, action]\n",
    "        q[state, action] = (1 - TERMINATION_PROB) * np.mean(\n",
    "            task.reward[state, action] + np.max(q[next_states, :], axis=1))\n",
    "\n",
    "        if step % eval_interval == 0:\n",
    "            v_pi = evaluate_pi(q, task)\n",
    "            performance.append([step, v_pi])\n",
    "\n",
    "    return zip(*performance)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# perform expected update from an on-policy distribution of the MDP @task\n",
    "# evaluate the learned q value every @eval_interval steps\n",
    "def on_policy(task, eval_interval):\n",
    "    performance = []\n",
    "    q = np.zeros((task.n_states, 2))\n",
    "    state = 0\n",
    "    for step in tqdm(range(MAX_STEPS)):\n",
    "        if np.random.rand() < EPSILON:\n",
    "            action = np.random.choice(ACTIONS)\n",
    "        else:\n",
    "            action = argmax(q[state])\n",
    "\n",
    "        next_state, _ = task.step(state, action)\n",
    "\n",
    "        next_states = task.transition[state, action]\n",
    "        q[state, action] = (1 - TERMINATION_PROB) * np.mean(\n",
    "            task.reward[state, action] + np.max(q[next_states, :], axis=1))\n",
    "\n",
    "        if next_state == task.n_states:\n",
    "            next_state = 0\n",
    "        state = next_state\n",
    "\n",
    "        if step % eval_interval == 0:\n",
    "            v_pi = evaluate_pi(q, task)\n",
    "            performance.append([step, v_pi])\n",
    "\n",
    "    return zip(*performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def figure_8_8():\n",
    "    num_states = [1000, 10000]\n",
    "    branch = [1, 3, 10]\n",
    "    methods = [on_policy, uniform]\n",
    "\n",
    "    # average across 30 tasks\n",
    "    n_tasks = 30\n",
    "\n",
    "    # number of evaluation points\n",
    "    x_ticks = 100\n",
    "\n",
    "    plt.figure(figsize=(10, 20))\n",
    "    for i, n in enumerate(num_states):\n",
    "        plt.subplot(2, 1, i+1)\n",
    "        for b in branch:\n",
    "            tasks = [Task(n, b) for _ in range(n_tasks)]\n",
    "            for method in methods:\n",
    "                steps = None\n",
    "                value = []\n",
    "                for task in tasks:\n",
    "                    steps, v = method(task, MAX_STEPS / x_ticks)\n",
    "                    value.append(v)\n",
    "                value = np.mean(np.asarray(value), axis=0)\n",
    "                plt.plot(steps, value, label=f'b = {b}, {method.__name__}')\n",
    "        plt.title(f'{n} states')\n",
    "\n",
    "        plt.ylabel('value of start state')\n",
    "        plt.legend()\n",
    "\n",
    "    plt.subplot(2, 1, 2)\n",
    "    plt.xlabel('computation time, in expected updates')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20000/20000 [00:28<00:00, 697.75it/s]\n",
      "100%|██████████| 20000/20000 [00:28<00:00, 690.21it/s]\n",
      "100%|██████████| 20000/20000 [00:29<00:00, 685.91it/s]\n",
      "100%|██████████| 20000/20000 [00:29<00:00, 679.08it/s]\n",
      "100%|██████████| 20000/20000 [00:29<00:00, 682.64it/s]\n",
      "100%|██████████| 20000/20000 [00:29<00:00, 678.51it/s]\n",
      "100%|██████████| 20000/20000 [00:29<00:00, 670.18it/s]\n",
      "100%|██████████| 20000/20000 [00:28<00:00, 694.41it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 796.78it/s]\n",
      "100%|██████████| 20000/20000 [00:24<00:00, 815.66it/s]\n",
      "100%|██████████| 20000/20000 [00:24<00:00, 807.22it/s]\n",
      "100%|██████████| 20000/20000 [00:24<00:00, 811.71it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 797.15it/s]\n",
      "100%|██████████| 20000/20000 [00:24<00:00, 815.00it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 785.03it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 772.24it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 797.70it/s]\n",
      "100%|██████████| 20000/20000 [00:25<00:00, 769.84it/s]\n",
      "100%|██████████| 20000/20000 [00:24<00:00, 804.41it/s]\n",
      " 11%|█         | 2200/20000 [00:02<00:22, 777.93it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfigure_8_8\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[9], line 21\u001b[0m, in \u001b[0;36mfigure_8_8\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m value \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m task \u001b[38;5;129;01min\u001b[39;00m tasks:\n\u001b[0;32m---> 21\u001b[0m     steps, v \u001b[38;5;241m=\u001b[39m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mMAX_STEPS\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mx_ticks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     22\u001b[0m     value\u001b[38;5;241m.\u001b[39mappend(v)\n\u001b[1;32m     23\u001b[0m value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mmean(np\u001b[38;5;241m.\u001b[39masarray(value), axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n",
      "Cell \u001b[0;32mIn[8], line 24\u001b[0m, in \u001b[0;36mon_policy\u001b[0;34m(task, eval_interval)\u001b[0m\n\u001b[1;32m     21\u001b[0m     state \u001b[38;5;241m=\u001b[39m next_state\n\u001b[1;32m     23\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m step \u001b[38;5;241m%\u001b[39m eval_interval \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 24\u001b[0m         v_pi \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate_pi\u001b[49m\u001b[43m(\u001b[49m\u001b[43mq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     25\u001b[0m         performance\u001b[38;5;241m.\u001b[39mappend([step, v_pi])\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mperformance)\n",
      "Cell \u001b[0;32mIn[5], line 12\u001b[0m, in \u001b[0;36mevaluate_pi\u001b[0;34m(q, task)\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m state \u001b[38;5;241m<\u001b[39m task\u001b[38;5;241m.\u001b[39mn_states:\n\u001b[1;32m     11\u001b[0m     action \u001b[38;5;241m=\u001b[39m argmax(q[state])\n\u001b[0;32m---> 12\u001b[0m     state, r \u001b[38;5;241m=\u001b[39m \u001b[43mtask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m     rewards \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m r\n\u001b[1;32m     14\u001b[0m returns\u001b[38;5;241m.\u001b[39mappend(rewards)\n",
      "Cell \u001b[0;32mIn[4], line 17\u001b[0m, in \u001b[0;36mTask.step\u001b[0;34m(self, state, action)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep\u001b[39m(\u001b[38;5;28mself\u001b[39m, state, action):\n\u001b[0;32m---> 17\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrandom\u001b[49m\u001b[38;5;241m.\u001b[39mrand() \u001b[38;5;241m<\u001b[39m TERMINATION_PROB:\n\u001b[1;32m     18\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_states, \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m     19\u001b[0m     next_ \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mb)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAALtCAYAAADnv8+CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl4klEQVR4nO3df2zX9Z3A8VdbbKuZrXgc5cfVcbpzblPBgfSqM8ZLZ5MZdvxxGYcLEKLz3DijNrsJ/qBzbpTbqSE5cUTmzv3jwWamWQbBcz3JsrMXMn4kmgOMYwxi1gK3s+XqRqX93B+L3XUUx7e2xfJ6PJLvH337fn8/7695iz79fPv9lhVFUQQAAEBS5Wd7AwAAAGeTKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFIrOYp+8pOfxIIFC2LGjBlRVlYWL7zwwh9ds3379vjkJz8ZVVVV8ZGPfCSeeeaZEWwVAABg9JUcRb29vTF79uxYv379Gc3/xS9+EbfcckvcdNNNsWfPnrjnnnvi9ttvjxdffLHkzQIAAIy2sqIoihEvLiuL559/PhYuXHjaOffdd19s2bIlXnvttcGxv/3bv4233nortm3bNtJLAwAAjIpJY32Bjo6OaGpqGjLW3Nwc99xzz2nXnDhxIk6cODH488DAQPz617+OP/mTP4mysrKx2ioAAPABVxRFHD9+PGbMmBHl5aPzEQljHkWdnZ1RV1c3ZKyuri56enriN7/5TZx//vmnrGlra4uHH354rLcGAABMUIcPH44/+7M/G5XnGvMoGolVq1ZFS0vL4M/d3d1xySWXxOHDh6OmpuYs7gwAADibenp6or6+Pi688MJRe84xj6Jp06ZFV1fXkLGurq6oqakZ9i5RRERVVVVUVVWdMl5TUyOKAACAUf21mjH/nqLGxsZob28fMvbSSy9FY2PjWF8aAADgjyo5iv73f/839uzZE3v27ImI333k9p49e+LQoUMR8bu3vi1dunRw/p133hkHDhyIr3zlK7Fv37548skn43vf+17ce++9o/MKAAAA3oeSo+hnP/tZXHPNNXHNNddERERLS0tcc801sXr16oiI+NWvfjUYSBERf/7nfx5btmyJl156KWbPnh2PPfZYfPvb347m5uZRegkAAAAj976+p2i89PT0RG1tbXR3d/udIgAASGws2mDMf6cIAADgg0wUAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASG1EUbR+/fqYNWtWVFdXR0NDQ+zYseM9569bty4++tGPxvnnnx/19fVx7733xm9/+9sRbRgAAGA0lRxFmzdvjpaWlmhtbY1du3bF7Nmzo7m5OY4cOTLs/GeffTZWrlwZra2tsXfv3nj66adj8+bNcf/997/vzQMAALxfJUfR448/Hl/4whdi+fLl8fGPfzw2bNgQF1xwQXznO98Zdv4rr7wS119/fdx6660xa9asuPnmm2Px4sV/9O4SAADAeCgpivr6+mLnzp3R1NT0+ycoL4+mpqbo6OgYds11110XO3fuHIygAwcOxNatW+Mzn/nMaa9z4sSJ6OnpGfIAAAAYC5NKmXzs2LHo7++Purq6IeN1dXWxb9++YdfceuutcezYsfjUpz4VRVHEyZMn484773zPt8+1tbXFww8/XMrWAAAARmTMP31u+/btsWbNmnjyySdj165d8YMf/CC2bNkSjzzyyGnXrFq1Krq7uwcfhw8fHuttAgAASZV0p2jKlClRUVERXV1dQ8a7urpi2rRpw6556KGHYsmSJXH77bdHRMRVV10Vvb29cccdd8QDDzwQ5eWndllVVVVUVVWVsjUAAIARKelOUWVlZcydOzfa29sHxwYGBqK9vT0aGxuHXfP222+fEj4VFRUREVEURan7BQAAGFUl3SmKiGhpaYlly5bFvHnzYv78+bFu3bro7e2N5cuXR0TE0qVLY+bMmdHW1hYREQsWLIjHH388rrnmmmhoaIg33ngjHnrooViwYMFgHAEAAJwtJUfRokWL4ujRo7F69ero7OyMOXPmxLZt2wY/fOHQoUND7gw9+OCDUVZWFg8++GC8+eab8ad/+qexYMGC+MY3vjF6rwIAAGCEyooJ8B62np6eqK2tje7u7qipqTnb2wEAAM6SsWiDMf/0OQAAgA8yUQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACA1UQQAAKQmigAAgNREEQAAkJooAgAAUhNFAABAaqIIAABITRQBAACpiSIAACC1EUXR+vXrY9asWVFdXR0NDQ2xY8eO95z/1ltvxYoVK2L69OlRVVUVl19+eWzdunVEGwYAABhNk0pdsHnz5mhpaYkNGzZEQ0NDrFu3Lpqbm2P//v0xderUU+b39fXFpz/96Zg6dWo899xzMXPmzPjlL38ZF1100WjsHwAA4H0pK4qiKGVBQ0NDXHvttfHEE09ERMTAwEDU19fHXXfdFStXrjxl/oYNG+Kf/umfYt++fXHeeeeNaJM9PT1RW1sb3d3dUVNTM6LnAAAAJr6xaIOS3j7X19cXO3fujKampt8/QXl5NDU1RUdHx7BrfvjDH0ZjY2OsWLEi6urq4sorr4w1a9ZEf3//aa9z4sSJ6OnpGfIAAAAYCyVF0bFjx6K/vz/q6uqGjNfV1UVnZ+ewaw4cOBDPPfdc9Pf3x9atW+Ohhx6Kxx57LL7+9a+f9jptbW1RW1s7+Kivry9lmwAAAGdszD99bmBgIKZOnRpPPfVUzJ07NxYtWhQPPPBAbNiw4bRrVq1aFd3d3YOPw4cPj/U2AQCApEr6oIUpU6ZERUVFdHV1DRnv6uqKadOmDbtm+vTpcd5550VFRcXg2Mc+9rHo7OyMvr6+qKysPGVNVVVVVFVVlbI1AACAESnpTlFlZWXMnTs32tvbB8cGBgaivb09Ghsbh11z/fXXxxtvvBEDAwODY6+//npMnz592CACAAAYTyW/fa6lpSU2btwY3/3ud2Pv3r3xxS9+MXp7e2P58uUREbF06dJYtWrV4PwvfvGL8etf/zruvvvueP3112PLli2xZs2aWLFixei9CgAAgBEq+XuKFi1aFEePHo3Vq1dHZ2dnzJkzJ7Zt2zb44QuHDh2K8vLft1Z9fX28+OKLce+998bVV18dM2fOjLvvvjvuu+++0XsVAAAAI1Ty9xSdDb6nCAAAiPgAfE8RAADAuUYUAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASG1EUbR+/fqYNWtWVFdXR0NDQ+zYseOM1m3atCnKyspi4cKFI7ksAADAqCs5ijZv3hwtLS3R2toau3btitmzZ0dzc3McOXLkPdcdPHgwvvzlL8cNN9ww4s0CAACMtpKj6PHHH48vfOELsXz58vj4xz8eGzZsiAsuuCC+853vnHZNf39/fP7zn4+HH344Lr300ve1YQAAgNFUUhT19fXFzp07o6mp6fdPUF4eTU1N0dHRcdp1X/va12Lq1Klx2223ndF1Tpw4ET09PUMeAAAAY6GkKDp27Fj09/dHXV3dkPG6urro7Owcds1Pf/rTePrpp2Pjxo1nfJ22traora0dfNTX15eyTQAAgDM2pp8+d/z48ViyZEls3LgxpkyZcsbrVq1aFd3d3YOPw4cPj+EuAQCAzCaVMnnKlClRUVERXV1dQ8a7urpi2rRpp8z/+c9/HgcPHowFCxYMjg0MDPzuwpMmxf79++Oyyy47ZV1VVVVUVVWVsjUAAIARKelOUWVlZcydOzfa29sHxwYGBqK9vT0aGxtPmX/FFVfEq6++Gnv27Bl8fPazn42bbrop9uzZ421xAADAWVfSnaKIiJaWlli2bFnMmzcv5s+fH+vWrYve3t5Yvnx5REQsXbo0Zs6cGW1tbVFdXR1XXnnlkPUXXXRRRMQp4wAAAGdDyVG0aNGiOHr0aKxevTo6Oztjzpw5sW3btsEPXzh06FCUl4/pryoBAACMmrKiKIqzvYk/pqenJ2pra6O7uztqamrO9nYAAICzZCzawC0dAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgtRFF0fr162PWrFlRXV0dDQ0NsWPHjtPO3bhxY9xwww0xefLkmDx5cjQ1Nb3nfAAAgPFUchRt3rw5WlpaorW1NXbt2hWzZ8+O5ubmOHLkyLDzt2/fHosXL46XX345Ojo6or6+Pm6++eZ488033/fmAQAA3q+yoiiKUhY0NDTEtddeG0888URERAwMDER9fX3cddddsXLlyj+6vr+/PyZPnhxPPPFELF269Iyu2dPTE7W1tdHd3R01NTWlbBcAADiHjEUblHSnqK+vL3bu3BlNTU2/f4Ly8mhqaoqOjo4zeo6333473nnnnbj44otPO+fEiRPR09Mz5AEAADAWSoqiY8eORX9/f9TV1Q0Zr6uri87OzjN6jvvuuy9mzJgxJKz+UFtbW9TW1g4+6uvrS9kmAADAGRvXT59bu3ZtbNq0KZ5//vmorq4+7bxVq1ZFd3f34OPw4cPjuEsAACCTSaVMnjJlSlRUVERXV9eQ8a6urpg2bdp7rn300Udj7dq18eMf/ziuvvrq95xbVVUVVVVVpWwNAABgREq6U1RZWRlz586N9vb2wbGBgYFob2+PxsbG06775je/GY888khs27Yt5s2bN/LdAgAAjLKS7hRFRLS0tMSyZcti3rx5MX/+/Fi3bl309vbG8uXLIyJi6dKlMXPmzGhra4uIiH/8x3+M1atXx7PPPhuzZs0a/N2jD33oQ/GhD31oFF8KAABA6UqOokWLFsXRo0dj9erV0dnZGXPmzIlt27YNfvjCoUOHorz89zegvvWtb0VfX1/8zd/8zZDnaW1tja9+9avvb/cAAADvU8nfU3Q2+J4iAAAg4gPwPUUAAADnGlEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgNVEEAACkJooAAIDURBEAAJCaKAIAAFITRQAAQGqiCAAASE0UAQAAqYkiAAAgtRFF0fr162PWrFlRXV0dDQ0NsWPHjvec//3vfz+uuOKKqK6ujquuuiq2bt06os0CAACMtpKjaPPmzdHS0hKtra2xa9eumD17djQ3N8eRI0eGnf/KK6/E4sWL47bbbovdu3fHwoULY+HChfHaa6+9780DAAC8X2VFURSlLGhoaIhrr702nnjiiYiIGBgYiPr6+rjrrrti5cqVp8xftGhR9Pb2xo9+9KPBsb/8y7+MOXPmxIYNG87omj09PVFbWxvd3d1RU1NTynYBAIBzyFi0waRSJvf19cXOnTtj1apVg2Pl5eXR1NQUHR0dw67p6OiIlpaWIWPNzc3xwgsvnPY6J06ciBMnTgz+3N3dHRG/+xsAAADk9W4TlHhv5z2VFEXHjh2L/v7+qKurGzJeV1cX+/btG3ZNZ2fnsPM7OztPe522trZ4+OGHTxmvr68vZbsAAMA56r//+7+jtrZ2VJ6rpCgaL6tWrRpyd+mtt96KD3/4w3Ho0KFRe+EwnJ6enqivr4/Dhw97qyZjylljvDhrjBdnjfHS3d0dl1xySVx88cWj9pwlRdGUKVOioqIiurq6hox3dXXFtGnThl0zbdq0kuZHRFRVVUVVVdUp47W1tf4hY1zU1NQ4a4wLZ43x4qwxXpw1xkt5+eh9u1BJz1RZWRlz586N9vb2wbGBgYFob2+PxsbGYdc0NjYOmR8R8dJLL512PgAAwHgq+e1zLS0tsWzZspg3b17Mnz8/1q1bF729vbF8+fKIiFi6dGnMnDkz2traIiLi7rvvjhtvvDEee+yxuOWWW2LTpk3xs5/9LJ566qnRfSUAAAAjUHIULVq0KI4ePRqrV6+Ozs7OmDNnTmzbtm3wwxQOHTo05FbWddddF88++2w8+OCDcf/998df/MVfxAsvvBBXXnnlGV+zqqoqWltbh31LHYwmZ43x4qwxXpw1xouzxngZi7NW8vcUAQAAnEtG77eTAAAAJiBRBAAApCaKAACA1EQRAACQ2gcmitavXx+zZs2K6urqaGhoiB07drzn/O9///txxRVXRHV1dVx11VWxdevWcdopE10pZ23jxo1xww03xOTJk2Py5MnR1NT0R88mvKvUP9fetWnTpigrK4uFCxeO7QY5Z5R61t56661YsWJFTJ8+PaqqquLyyy/371HOSKlnbd26dfHRj340zj///Kivr4977703fvvb347TbpmIfvKTn8SCBQtixowZUVZWFi+88MIfXbN9+/b45Cc/GVVVVfGRj3wknnnmmZKv+4GIos2bN0dLS0u0trbGrl27Yvbs2dHc3BxHjhwZdv4rr7wSixcvjttuuy12794dCxcujIULF8Zrr702zjtnoin1rG3fvj0WL14cL7/8cnR0dER9fX3cfPPN8eabb47zzploSj1r7zp48GB8+ctfjhtuuGGcdspEV+pZ6+vri09/+tNx8ODBeO6552L//v2xcePGmDlz5jjvnImm1LP27LPPxsqVK6O1tTX27t0bTz/9dGzevDnuv//+cd45E0lvb2/Mnj071q9ff0bzf/GLX8Qtt9wSN910U+zZsyfuueeeuP322+PFF18s7cLFB8D8+fOLFStWDP7c399fzJgxo2hraxt2/uc+97nilltuGTLW0NBQ/N3f/d2Y7pOJr9Sz9odOnjxZXHjhhcV3v/vdsdoi54iRnLWTJ08W1113XfHtb3+7WLZsWfHXf/3X47BTJrpSz9q3vvWt4tJLLy36+vrGa4ucI0o9aytWrCj+6q/+ashYS0tLcf3114/pPjl3RETx/PPPv+ecr3zlK8UnPvGJIWOLFi0qmpubS7rWWb9T1NfXFzt37oympqbBsfLy8mhqaoqOjo5h13R0dAyZHxHR3Nx82vkQMbKz9ofefvvteOedd+Liiy8eq21yDhjpWfva174WU6dOjdtuu208tsk5YCRn7Yc//GE0NjbGihUroq6uLq688spYs2ZN9Pf3j9e2mYBGctauu+662Llz5+Bb7A4cOBBbt26Nz3zmM+OyZ3IYrS6YNJqbGoljx45Ff39/1NXVDRmvq6uLffv2Dbums7Nz2PmdnZ1jtk8mvpGctT903333xYwZM075hw/+v5GctZ/+9Kfx9NNPx549e8Zhh5wrRnLWDhw4EP/+7/8en//852Pr1q3xxhtvxJe+9KV45513orW1dTy2zQQ0krN26623xrFjx+JTn/pUFEURJ0+ejDvvvNPb5xhVp+uCnp6e+M1vfhPnn3/+GT3PWb9TBBPF2rVrY9OmTfH8889HdXX12d4O55Djx4/HkiVLYuPGjTFlypSzvR3OcQMDAzF16tR46qmnYu7cubFo0aJ44IEHYsOGDWd7a5xjtm/fHmvWrIknn3wydu3aFT/4wQ9iy5Yt8cgjj5ztrcEpzvqdoilTpkRFRUV0dXUNGe/q6opp06YNu2batGklzYeIkZ21dz366KOxdu3a+PGPfxxXX331WG6Tc0CpZ+3nP/95HDx4MBYsWDA4NjAwEBERkyZNiv3798dll102tptmQhrJn2vTp0+P8847LyoqKgbHPvaxj0VnZ2f09fVFZWXlmO6ZiWkkZ+2hhx6KJUuWxO233x4REVdddVX09vbGHXfcEQ888ECUl/t/87x/p+uCmpqaM75LFPEBuFNUWVkZc+fOjfb29sGxgYGBaG9vj8bGxmHXNDY2DpkfEfHSSy+ddj5EjOysRUR885vfjEceeSS2bdsW8+bNG4+tMsGVetauuOKKePXVV2PPnj2Dj89+9rODn6RTX18/nttnAhnJn2vXX399vPHGG4PhHRHx+uuvx/Tp0wURpzWSs/b222+fEj7vxvjvfoce3r9R64LSPgNibGzatKmoqqoqnnnmmeK//uu/ijvuuKO46KKLis7OzqIoimLJkiXFypUrB+f/x3/8RzFp0qTi0UcfLfbu3Vu0trYW5513XvHqq6+erZfABFHqWVu7dm1RWVlZPPfcc8WvfvWrwcfx48fP1ktggij1rP0hnz7HmSr1rB06dKi48MILi7//+78v9u/fX/zoRz8qpk6dWnz9618/Wy+BCaLUs9ba2lpceOGFxb/+678WBw4cKP7t3/6tuOyyy4rPfe5zZ+slMAEcP3682L17d7F79+4iIorHH3+82L17d/HLX/6yKIqiWLlyZbFkyZLB+QcOHCguuOCC4h/+4R+KvXv3FuvXry8qKiqKbdu2lXTdD0QUFUVR/PM//3NxySWXFJWVlcX8+fOL//zP/xz8azfeeGOxbNmyIfO/973vFZdffnlRWVlZfOITnyi2bNkyzjtmoirlrH34wx8uIuKUR2tr6/hvnAmn1D/X/j9RRClKPWuvvPJK0dDQUFRVVRWXXnpp8Y1vfKM4efLkOO+aiaiUs/bOO+8UX/3qV4vLLrusqK6uLurr64svfelLxf/8z/+M/8aZMF5++eVh/9vr3bO1bNmy4sYbbzxlzZw5c4rKysri0ksvLf7lX/6l5OuWFYX7lwAAQF5n/XeKAAAAziZRBAAApCaKAACA1EQRAACQmigCAABSE0UAAEBqoggAAEhNFAEAAKmJIgAAIDVRBAAApCaKAACA1EQRAACQ2v8BsBcAl2lQdZUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x2000 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figure_8_8()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
